package com.morpx.sim.crawl;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLConnection;
import java.security.MessageDigest;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.google.gson.Gson;
import com.morpx.sim.apps.CrawlerThread;
import com.morpx.sim.storage.Item;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.BinaryParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import edu.uci.ics.crawler4j.util.IO;

// The VipItem data structure correspond to the json data format embedded in 
// vip.com page.
class VipItem {
  public String link;
  public String name;
  public String img;
  public String brand;
  public String gender;
  public String age;
  public float sell_price;
  public float market_price;
  
  // Constant to group useful information embedded in html.
  private static final Pattern CONTENT_GROUPER =
//      Pattern.compile("title=\"(.*?)\".*src=\"(http:\\/\\/a.vpimg[0-9].com\\/upload\\/merchandise\\/.*?.jpg)\".*\\&yen;(.*?)<.*\\&yen;(.*?)<",
//        Pattern.DOTALL);
      Pattern.compile("<a href=\"(\\/detail-[0-9]*-[0-9]*\\.html)\".*title=\"(.*?)\".*src=\"(http:\\/\\/a.vpimg[0-9].com\\/upload\\/merchandise\\/.*?.jpg)\".*<dd class=\"brand_tit gray\"> *?(.*?) *?<\\/dd>.*\\&yen;(.*?)<.*\\&yen;(.*?)<",
          Pattern.DOTALL);
  
  // Parse the data from HTML tags.
  public static VipItem parseItemFromHTML(String itemBlock) {
    Matcher m = CONTENT_GROUPER.matcher(itemBlock);
    if (m.find()) {
      VipItem item = new VipItem();
      item.link = m.group(1);
      item.name = m.group(2);
      item.img = m.group(3);
      item.brand = m.group(4);
      item.sell_price = Float.valueOf(m.group(5));
      item.market_price = Float.valueOf(m.group(6));
      return item;
    } else {
      return null;
    }
  }
}

/**
 * The crawler that parses the vip.com's sale page and store all the products.
 * @author tianli
 */
public class VipCrawler extends WebCrawler {
  public static final int MAX_CRAWLING_DEPTH = 3;
  public static final int IMAGE_FETCHING_TIMEOUT = 10000;

  private static final Pattern LINK_FILTER = Pattern.compile(".*(\\.(css|js|mid|mp2|mp3|mp4|wav|avi|mov|mpeg|ram|m4v|pdf"
                  + "|rm|smil|wmv|swf|wma|zip|rar|gz|bmp|gif|jpe?g|png|tiff?))$");
  
  private static final Pattern PRELOAD_PRODUCT_PATTERN =
      Pattern.compile("<dl class=\"J_pro_items\"(.*?)<\\/dl>",
                      Pattern.DOTALL);
  
  private static final Pattern LAZYLOAD_PRODUCT_PATTERN =
      Pattern.compile("\\{\"id\".*?\"img\" : \"http:\\/\\/a.vpimg.*?\\}");
  
  private static final String ID_PREFIX = "vip";
  
  private static File storageFolder;
  
  private static final Pattern VIP_SHOW_PATTERN = 
      Pattern.compile(".*\\.com/show-[0-9]*\\.html(\\?(q=0%7c0%7c0%7c0.*))?");
  private static final String VIP_SHOW_FIRST_PAGE_SUFFIX = "%7c1";
  
  private static final Pattern VIP_SHOW_TITLE_PATTERN =
      Pattern.compile("<title>(.*)<\\/title>");
  
  private static Pattern VIP_KEYWORDS =
      Pattern.compile("鞋|女装|男装|男女|童装|裙|衣|裤");
  
  private static final Pattern VIP_ITEM_LINK_PATTERN = 
      Pattern.compile("\\/detail-([0-9]*)-([0-9]*)\\.html");
  
  // gender check
  private static enum GENDER {UNCERTAIN, FEMALE, MALE, BOTH;
  	@Override
  	public String toString() {
  		String gender = null;
  		switch (this) {
  		case BOTH: gender = "男女"; break;
  		case FEMALE: gender = "女"; break;
  		case MALE: gender = "男"; break;
  		case UNCERTAIN: gender = null; break;
  		default:
  			break;
  		}
  		return gender;
  	};
  };
  private static GENDER getGender(String str) {
  	if (str.contains("男") && str.contains("女")) return GENDER.BOTH;
  	else if (str.contains("男") && !str.contains("女")) return GENDER.MALE;
  	else if (!str.contains("男") && str.contains("女")) return GENDER.FEMALE;
  	else if (!str.contains("男") && !str.contains("女")) return GENDER.UNCERTAIN;
		return GENDER.UNCERTAIN;
  }

  public static void configure(String storageFolderName, String keywords) {
    if (keywords != null)
      VIP_KEYWORDS = Pattern.compile(keywords);
    storageFolder = new File(storageFolderName);
    if (!storageFolder.exists()) {
      storageFolder.mkdirs();
    }
  }

  // Only visit the top level show pages and all the next page links.
  @Override
  public boolean shouldVisit(WebURL url) {
    String href = url.getURL().toLowerCase();
    if (LINK_FILTER.matcher(href).matches()) {
      return false;
    }

    if (VIP_SHOW_PATTERN.matcher(href).matches()) {
      if (href.endsWith(VIP_SHOW_FIRST_PAGE_SUFFIX)) {
        // The first page is crawled when we enter the show, so do not crawl
        // again.
        return false;
      }
      //System.out.println("should visit " + href);
      return true;
    }

    return false;
  }
  
  // Parse and save the contents on the page.
  @Override
  public void visit(Page page) {
    String url = page.getWebURL().getURL();
    Gson gson = new Gson();
    int foundItems = 0;
    
    try {
      String content = new String(page.getContentData(),
        page.getContentCharset());
      System.err.println("Visiting " + url);
      
      // Check the title tag.
      Matcher titleMatcher = VIP_SHOW_TITLE_PATTERN.matcher(content);
      boolean categoryMatch = false;
      GENDER categoryGender = GENDER.UNCERTAIN;
      if (titleMatcher.find()) {
        System.err.println("Show title: " + titleMatcher.group());

        Matcher categoryMatcher = VIP_KEYWORDS.matcher(titleMatcher.group());
        if (categoryMatcher.find()) {
          categoryMatch = true;
          categoryGender = getGender(titleMatcher.group());
        }
      }
      
      if (!categoryMatch) {
        System.err.println("Category not match, stop.");
        return;
      }

      String brand = null;
      // Find all the items on the page if the description contains some keywords.
      Matcher matcher = PRELOAD_PRODUCT_PATTERN.matcher(content);
      while (matcher.find()) {
        String block = matcher.group(1);
        //System.out.println("Found match " + " at " +
        //                   matcher.start() + "->" + matcher.end());
        VipItem item = VipItem.parseItemFromHTML(block);
        if (item != null) {
        	GENDER itemGender = getGender(item.name);
        	if (itemGender == GENDER.UNCERTAIN)
        		item.gender = categoryGender.toString();
          System.err.println("Fetching product from " + item.img);
          fetchItemToStorage(item);
          foundItems++;

          // get brand
          if (brand == null || brand.equals("")) {
          	brand = item.brand;
          }
        }
      }
      
      if (brand == null)
      	brand = "";
      matcher = LAZYLOAD_PRODUCT_PATTERN.matcher(content);
      while (matcher.find()) {
        String json = matcher.group();
        //System.out.println(json);
        
        VipItem item = gson.fromJson(json, VipItem.class);
        if (item != null) {
        	GENDER itemGender = getGender(item.name);
        	if (itemGender == GENDER.UNCERTAIN)
        		item.gender = categoryGender.toString();
        	item.brand = brand;
          System.err.println("Fetching product from " + item.img);
          fetchItemToStorage(item);
          foundItems++;
        }
      }
    
    } catch (UnsupportedEncodingException e) {
      e.printStackTrace();
    }
    System.err.println("Saved " + foundItems + " items.");
  }
  
  // Fetch an item image and generate related json, save it in the storage.
  private void fetchItemToStorage(VipItem item) {
    String[] splittedURL = item.img.split("\\/");
    String[] splittedFileName = splittedURL[splittedURL.length - 1].split("\\.");
    
    Matcher linkMatcher = VIP_ITEM_LINK_PATTERN.matcher(item.link);
    if (!linkMatcher.find() || linkMatcher.groupCount() != 2) {
      System.err.println("Wrong link cannot match: " + item.link);
      return;
    }
    String image_id = ID_PREFIX + '-' + linkMatcher.group(1) + '-' + linkMatcher.group(2);
    // Generates the morpx item.
    Item morpxItem = new Item();
//    String image_id = ID_PREFIX + 
//        "_" + splittedURL[splittedURL.length - 2] +
//        "_" + splittedFileName[0];
    morpxItem.imagePaths = new String[1];
    morpxItem.imagePaths[0] = image_id;
    if (splittedFileName.length > 1) {
      morpxItem.imagePaths[0] += "." + splittedFileName[splittedFileName.length - 1];	// use the last string
    }
    morpxItem.imageURLs = new String[1];
    morpxItem.imageURLs[0] = item.img;
    morpxItem.title = item.name;
    morpxItem.tags = morpxItem.title + ((item.gender == null || item.gender.equals("")) ? "" : "(" + item.gender + ")");
    morpxItem.brand = item.brand;

    String outputPath = storageFolder.getAbsolutePath() + "/" +
        morpxItem.imagePaths[0];
    String outputJsonPath = storageFolder.getAbsolutePath() + "/" + image_id +
        ".json";
    if (splittedURL.length > 2) {
      try {
        URL url = new URL(item.img);
        URLConnection connection = url.openConnection();
        connection.setConnectTimeout(IMAGE_FETCHING_TIMEOUT);
        connection.setReadTimeout(IMAGE_FETCHING_TIMEOUT);
        BufferedInputStream bin = new BufferedInputStream(
          connection.getInputStream());
        BufferedOutputStream bout = new BufferedOutputStream(
          new FileOutputStream(outputPath));
        int nRead;
        byte[] data = new byte[16384];

        while ((nRead = bin.read(data, 0, data.length)) != -1) {
          bout.write(data, 0, nRead);
        }
        bout.close();
        
        // Also save json.
        Gson gson = new Gson();
        String json = gson.toJson(morpxItem);
        FileWriter jout = new FileWriter(outputJsonPath);
        jout.write(json);
        jout.close();
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
  }
}
